import scrapy
from items import DragonrajaItem


class MainSpider(scrapy.Spider):
    name = 'main'
    start_urls = ['https://www.shizongzui.cc/longzu/']
    urls = []

    def parse(self, response, **kwargs):
        global page, charpter
        urls = []
        span_list = response.xpath(
            '/html/body/div[@class="booklist clearfix"]/span')
        for span in span_list:
            Ischapter = span.xpath('./@class')
            if len(Ischapter) != 0:
                charpter = span.xpath('./text()').extract_first().strip()
                # print(charpter)
            else:
                page = span.xpath('./a/text()').extract_first().strip()
                url = span.xpath('./a/@href').extract_first().strip()
                # print(page)
                self.urls.append(
                    {'url': url, 'title': page, 'charpter': charpter})
        for url in self.urls:
            yield scrapy.Request(url=url['url'], callback=self.analyze, meta={'title': url['title'], 'charpter': url['charpter']})

    def analyze(self, response):
        item = DragonrajaItem()
        item['title'] = response.meta['title']
        item['charpter'] = response.meta['charpter']
        contents = response.xpath(
            '//*[@id="BookText"]/text()').extract()
        content = '\n'.join(contents)
        item['content'] = content
        yield item
